{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /home/jupyter/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "#%matplotlib qt\n",
    "%matplotlib inline\n",
    "import os\n",
    "import sys\n",
    "import numpy as np\n",
    "import torch\n",
    "import matplotlib.pyplot as plt\n",
    "import json\n",
    "import torchtext\n",
    "import torch.nn as nn\n",
    "import nltk\n",
    "import pandas as pd\n",
    "\n",
    "nltk.download('punkt')\n",
    "cmap = plt.get_cmap('viridis')\n",
    "from mpl_toolkits.mplot3d import Axes3D"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "rootdir = '/home/stephane/automatic-data-generation/'\n",
    "dataroot= rootdir + 'data/'\n",
    "os.chdir(rootdir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Grid search 1 : general"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "python -m grid results_grid \"python automatic_data_generation/train_and_eval_cvae.py -ng 2000 -ep 50\" --dataset-size:int 125 250 500 1000 --seed:int 1 2 3 --temperature:float 0.5 1 2 --embedding-dimension:int 100 300 --hidden-size:int 256 512 --n_parallel 8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "36\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pickle</th>\n",
       "      <th>dataset_size</th>\n",
       "      <th>n_epochs</th>\n",
       "      <th>seed</th>\n",
       "      <th>hidden_size</th>\n",
       "      <th>embedding_dimension</th>\n",
       "      <th>temperature</th>\n",
       "      <th>bleu_quality</th>\n",
       "      <th>bleu_diversity</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>originality</th>\n",
       "      <th>transfer</th>\n",
       "      <th>metric</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>42754</td>\n",
       "      <td>125</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>512</td>\n",
       "      <td>300</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.673257</td>\n",
       "      <td>0.034804</td>\n",
       "      <td>0.9380</td>\n",
       "      <td>0.222829</td>\n",
       "      <td>0.386131</td>\n",
       "      <td>0.117490</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>22314</td>\n",
       "      <td>125</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>512</td>\n",
       "      <td>300</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.672509</td>\n",
       "      <td>0.034813</td>\n",
       "      <td>0.9690</td>\n",
       "      <td>0.144285</td>\n",
       "      <td>0.290224</td>\n",
       "      <td>0.105307</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>13588</td>\n",
       "      <td>125</td>\n",
       "      <td>50</td>\n",
       "      <td>3</td>\n",
       "      <td>512</td>\n",
       "      <td>300</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.678157</td>\n",
       "      <td>0.063655</td>\n",
       "      <td>0.9440</td>\n",
       "      <td>0.168844</td>\n",
       "      <td>0.329153</td>\n",
       "      <td>0.094887</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>14347</td>\n",
       "      <td>125</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>512</td>\n",
       "      <td>100</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.667885</td>\n",
       "      <td>0.030559</td>\n",
       "      <td>0.9420</td>\n",
       "      <td>0.163037</td>\n",
       "      <td>0.512630</td>\n",
       "      <td>0.089700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>49592</td>\n",
       "      <td>125</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>256</td>\n",
       "      <td>100</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.612760</td>\n",
       "      <td>0.053460</td>\n",
       "      <td>0.9325</td>\n",
       "      <td>0.160250</td>\n",
       "      <td>0.567055</td>\n",
       "      <td>0.079669</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>24607</td>\n",
       "      <td>125</td>\n",
       "      <td>50</td>\n",
       "      <td>3</td>\n",
       "      <td>256</td>\n",
       "      <td>100</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.616589</td>\n",
       "      <td>0.061802</td>\n",
       "      <td>0.9520</td>\n",
       "      <td>0.129616</td>\n",
       "      <td>0.467620</td>\n",
       "      <td>0.079255</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>38011</td>\n",
       "      <td>125</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>512</td>\n",
       "      <td>100</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.607011</td>\n",
       "      <td>0.063375</td>\n",
       "      <td>0.9205</td>\n",
       "      <td>0.169388</td>\n",
       "      <td>0.529006</td>\n",
       "      <td>0.073981</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>34433</td>\n",
       "      <td>125</td>\n",
       "      <td>50</td>\n",
       "      <td>3</td>\n",
       "      <td>512</td>\n",
       "      <td>100</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.571414</td>\n",
       "      <td>0.079712</td>\n",
       "      <td>0.8930</td>\n",
       "      <td>0.228991</td>\n",
       "      <td>0.525362</td>\n",
       "      <td>0.073847</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>32580</td>\n",
       "      <td>125</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>256</td>\n",
       "      <td>300</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.687051</td>\n",
       "      <td>0.030944</td>\n",
       "      <td>0.9530</td>\n",
       "      <td>0.108083</td>\n",
       "      <td>0.437325</td>\n",
       "      <td>0.066786</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>26031</td>\n",
       "      <td>125</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>512</td>\n",
       "      <td>300</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.694310</td>\n",
       "      <td>0.036054</td>\n",
       "      <td>0.9575</td>\n",
       "      <td>0.096807</td>\n",
       "      <td>0.385464</td>\n",
       "      <td>0.062704</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>00792</td>\n",
       "      <td>125</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>512</td>\n",
       "      <td>100</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.688911</td>\n",
       "      <td>0.046566</td>\n",
       "      <td>0.9420</td>\n",
       "      <td>0.112937</td>\n",
       "      <td>0.576502</td>\n",
       "      <td>0.062137</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>46737</td>\n",
       "      <td>125</td>\n",
       "      <td>50</td>\n",
       "      <td>3</td>\n",
       "      <td>512</td>\n",
       "      <td>300</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.630650</td>\n",
       "      <td>0.056326</td>\n",
       "      <td>0.9665</td>\n",
       "      <td>0.081394</td>\n",
       "      <td>0.405602</td>\n",
       "      <td>0.057891</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>25969</td>\n",
       "      <td>125</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>256</td>\n",
       "      <td>100</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.611492</td>\n",
       "      <td>0.058003</td>\n",
       "      <td>0.9110</td>\n",
       "      <td>0.142592</td>\n",
       "      <td>0.596000</td>\n",
       "      <td>0.056141</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>21906</td>\n",
       "      <td>125</td>\n",
       "      <td>50</td>\n",
       "      <td>3</td>\n",
       "      <td>256</td>\n",
       "      <td>300</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.636611</td>\n",
       "      <td>0.069972</td>\n",
       "      <td>0.9515</td>\n",
       "      <td>0.092129</td>\n",
       "      <td>0.469503</td>\n",
       "      <td>0.056038</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>13701</td>\n",
       "      <td>125</td>\n",
       "      <td>50</td>\n",
       "      <td>3</td>\n",
       "      <td>512</td>\n",
       "      <td>100</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.570829</td>\n",
       "      <td>0.090768</td>\n",
       "      <td>0.9405</td>\n",
       "      <td>0.102076</td>\n",
       "      <td>0.536663</td>\n",
       "      <td>0.055273</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>30959</td>\n",
       "      <td>125</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>256</td>\n",
       "      <td>300</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.696726</td>\n",
       "      <td>0.027088</td>\n",
       "      <td>0.9555</td>\n",
       "      <td>0.085726</td>\n",
       "      <td>0.492974</td>\n",
       "      <td>0.054377</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>47896</td>\n",
       "      <td>125</td>\n",
       "      <td>50</td>\n",
       "      <td>3</td>\n",
       "      <td>512</td>\n",
       "      <td>300</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.640139</td>\n",
       "      <td>0.043642</td>\n",
       "      <td>0.9695</td>\n",
       "      <td>0.068022</td>\n",
       "      <td>0.343276</td>\n",
       "      <td>0.049903</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>32896</td>\n",
       "      <td>125</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>256</td>\n",
       "      <td>100</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.662086</td>\n",
       "      <td>0.046815</td>\n",
       "      <td>0.9180</td>\n",
       "      <td>0.116158</td>\n",
       "      <td>0.598224</td>\n",
       "      <td>0.049372</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>14501</td>\n",
       "      <td>125</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>512</td>\n",
       "      <td>300</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.705872</td>\n",
       "      <td>0.030835</td>\n",
       "      <td>0.9645</td>\n",
       "      <td>0.069288</td>\n",
       "      <td>0.372186</td>\n",
       "      <td>0.048270</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>45221</td>\n",
       "      <td>125</td>\n",
       "      <td>50</td>\n",
       "      <td>3</td>\n",
       "      <td>512</td>\n",
       "      <td>100</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.626971</td>\n",
       "      <td>0.072697</td>\n",
       "      <td>0.9535</td>\n",
       "      <td>0.077698</td>\n",
       "      <td>0.404635</td>\n",
       "      <td>0.048263</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>42587</td>\n",
       "      <td>125</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>512</td>\n",
       "      <td>100</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.608808</td>\n",
       "      <td>0.046313</td>\n",
       "      <td>0.9275</td>\n",
       "      <td>0.102409</td>\n",
       "      <td>0.541114</td>\n",
       "      <td>0.048248</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>47638</td>\n",
       "      <td>125</td>\n",
       "      <td>50</td>\n",
       "      <td>3</td>\n",
       "      <td>256</td>\n",
       "      <td>100</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.578973</td>\n",
       "      <td>0.083549</td>\n",
       "      <td>0.9010</td>\n",
       "      <td>0.136180</td>\n",
       "      <td>0.604176</td>\n",
       "      <td>0.048013</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>05658</td>\n",
       "      <td>125</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>512</td>\n",
       "      <td>100</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.699813</td>\n",
       "      <td>0.023496</td>\n",
       "      <td>0.9570</td>\n",
       "      <td>0.073301</td>\n",
       "      <td>0.485926</td>\n",
       "      <td>0.047231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>37390</td>\n",
       "      <td>125</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>256</td>\n",
       "      <td>300</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.711728</td>\n",
       "      <td>0.033026</td>\n",
       "      <td>0.9550</td>\n",
       "      <td>0.068620</td>\n",
       "      <td>0.430379</td>\n",
       "      <td>0.043300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>21365</td>\n",
       "      <td>125</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>256</td>\n",
       "      <td>100</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.621369</td>\n",
       "      <td>0.072725</td>\n",
       "      <td>0.8860</td>\n",
       "      <td>0.145168</td>\n",
       "      <td>0.599809</td>\n",
       "      <td>0.043272</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>47457</td>\n",
       "      <td>125</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>512</td>\n",
       "      <td>100</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.660434</td>\n",
       "      <td>0.053532</td>\n",
       "      <td>0.9580</td>\n",
       "      <td>0.065404</td>\n",
       "      <td>0.444304</td>\n",
       "      <td>0.042585</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>00950</td>\n",
       "      <td>125</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>256</td>\n",
       "      <td>300</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.663454</td>\n",
       "      <td>0.048124</td>\n",
       "      <td>0.9210</td>\n",
       "      <td>0.093442</td>\n",
       "      <td>0.569648</td>\n",
       "      <td>0.041033</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>24147</td>\n",
       "      <td>125</td>\n",
       "      <td>50</td>\n",
       "      <td>3</td>\n",
       "      <td>256</td>\n",
       "      <td>300</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.618193</td>\n",
       "      <td>0.078621</td>\n",
       "      <td>0.9565</td>\n",
       "      <td>0.063990</td>\n",
       "      <td>0.431036</td>\n",
       "      <td>0.041017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>29837</td>\n",
       "      <td>125</td>\n",
       "      <td>50</td>\n",
       "      <td>3</td>\n",
       "      <td>256</td>\n",
       "      <td>300</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.614358</td>\n",
       "      <td>0.088901</td>\n",
       "      <td>0.8950</td>\n",
       "      <td>0.119255</td>\n",
       "      <td>0.610126</td>\n",
       "      <td>0.039329</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>31492</td>\n",
       "      <td>125</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>256</td>\n",
       "      <td>300</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.680360</td>\n",
       "      <td>0.044448</td>\n",
       "      <td>0.9525</td>\n",
       "      <td>0.063096</td>\n",
       "      <td>0.406306</td>\n",
       "      <td>0.038784</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>37913</td>\n",
       "      <td>125</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>256</td>\n",
       "      <td>300</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.692112</td>\n",
       "      <td>0.033365</td>\n",
       "      <td>0.9455</td>\n",
       "      <td>0.060561</td>\n",
       "      <td>0.470918</td>\n",
       "      <td>0.034579</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00981</td>\n",
       "      <td>125</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>256</td>\n",
       "      <td>100</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.560037</td>\n",
       "      <td>0.085395</td>\n",
       "      <td>0.8220</td>\n",
       "      <td>0.217350</td>\n",
       "      <td>0.685548</td>\n",
       "      <td>0.030611</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>00849</td>\n",
       "      <td>125</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>512</td>\n",
       "      <td>300</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.673394</td>\n",
       "      <td>0.023616</td>\n",
       "      <td>0.9820</td>\n",
       "      <td>0.032235</td>\n",
       "      <td>0.233880</td>\n",
       "      <td>0.026880</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>21313</td>\n",
       "      <td>125</td>\n",
       "      <td>50</td>\n",
       "      <td>3</td>\n",
       "      <td>256</td>\n",
       "      <td>100</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.545457</td>\n",
       "      <td>0.109179</td>\n",
       "      <td>0.8015</td>\n",
       "      <td>0.239234</td>\n",
       "      <td>0.684902</td>\n",
       "      <td>0.026173</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>38614</td>\n",
       "      <td>125</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>256</td>\n",
       "      <td>100</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.572596</td>\n",
       "      <td>0.066132</td>\n",
       "      <td>0.7820</td>\n",
       "      <td>0.256967</td>\n",
       "      <td>0.677747</td>\n",
       "      <td>0.021976</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>38660</td>\n",
       "      <td>125</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>512</td>\n",
       "      <td>300</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.648340</td>\n",
       "      <td>0.043398</td>\n",
       "      <td>0.9910</td>\n",
       "      <td>0.022655</td>\n",
       "      <td>0.227202</td>\n",
       "      <td>0.020697</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   pickle  dataset_size  n_epochs  seed  hidden_size  embedding_dimension  \\\n",
       "23  42754           125        50     2          512                  300   \n",
       "7   22314           125        50     1          512                  300   \n",
       "15  13588           125        50     3          512                  300   \n",
       "16  14347           125        50     2          512                  100   \n",
       "19  49592           125        50     2          256                  100   \n",
       "2   24607           125        50     3          256                  100   \n",
       "6   38011           125        50     1          512                  100   \n",
       "35  34433           125        50     3          512                  100   \n",
       "29  32580           125        50     2          256                  300   \n",
       "31  26031           125        50     2          512                  300   \n",
       "27  00792           125        50     2          512                  100   \n",
       "30  46737           125        50     3          512                  300   \n",
       "1   25969           125        50     1          256                  100   \n",
       "33  21906           125        50     3          256                  300   \n",
       "10  13701           125        50     3          512                  100   \n",
       "14  30959           125        50     2          256                  300   \n",
       "34  47896           125        50     3          512                  300   \n",
       "8   32896           125        50     2          256                  100   \n",
       "13  14501           125        50     2          512                  300   \n",
       "5   45221           125        50     3          512                  100   \n",
       "12  42587           125        50     1          512                  100   \n",
       "9   47638           125        50     3          256                  100   \n",
       "4   05658           125        50     2          512                  100   \n",
       "24  37390           125        50     1          256                  300   \n",
       "3   21365           125        50     1          256                  100   \n",
       "25  47457           125        50     1          512                  100   \n",
       "20  00950           125        50     1          256                  300   \n",
       "21  24147           125        50     3          256                  300   \n",
       "11  29837           125        50     3          256                  300   \n",
       "22  31492           125        50     1          256                  300   \n",
       "32  37913           125        50     2          256                  300   \n",
       "0   00981           125        50     1          256                  100   \n",
       "18  00849           125        50     1          512                  300   \n",
       "26  21313           125        50     3          256                  100   \n",
       "28  38614           125        50     2          256                  100   \n",
       "17  38660           125        50     1          512                  300   \n",
       "\n",
       "    temperature  bleu_quality  bleu_diversity  accuracy  originality  \\\n",
       "23          0.5      0.673257        0.034804    0.9380     0.222829   \n",
       "7           0.5      0.672509        0.034813    0.9690     0.144285   \n",
       "15          0.5      0.678157        0.063655    0.9440     0.168844   \n",
       "16          0.5      0.667885        0.030559    0.9420     0.163037   \n",
       "19          0.5      0.612760        0.053460    0.9325     0.160250   \n",
       "2           0.5      0.616589        0.061802    0.9520     0.129616   \n",
       "6           0.5      0.607011        0.063375    0.9205     0.169388   \n",
       "35          0.5      0.571414        0.079712    0.8930     0.228991   \n",
       "29          0.5      0.687051        0.030944    0.9530     0.108083   \n",
       "31          2.0      0.694310        0.036054    0.9575     0.096807   \n",
       "27          1.0      0.688911        0.046566    0.9420     0.112937   \n",
       "30          1.0      0.630650        0.056326    0.9665     0.081394   \n",
       "1           0.5      0.611492        0.058003    0.9110     0.142592   \n",
       "33          0.5      0.636611        0.069972    0.9515     0.092129   \n",
       "10          2.0      0.570829        0.090768    0.9405     0.102076   \n",
       "14          1.0      0.696726        0.027088    0.9555     0.085726   \n",
       "34          2.0      0.640139        0.043642    0.9695     0.068022   \n",
       "8           1.0      0.662086        0.046815    0.9180     0.116158   \n",
       "13          1.0      0.705872        0.030835    0.9645     0.069288   \n",
       "5           1.0      0.626971        0.072697    0.9535     0.077698   \n",
       "12          1.0      0.608808        0.046313    0.9275     0.102409   \n",
       "9           1.0      0.578973        0.083549    0.9010     0.136180   \n",
       "4           2.0      0.699813        0.023496    0.9570     0.073301   \n",
       "24          0.5      0.711728        0.033026    0.9550     0.068620   \n",
       "3           1.0      0.621369        0.072725    0.8860     0.145168   \n",
       "25          2.0      0.660434        0.053532    0.9580     0.065404   \n",
       "20          2.0      0.663454        0.048124    0.9210     0.093442   \n",
       "21          1.0      0.618193        0.078621    0.9565     0.063990   \n",
       "11          2.0      0.614358        0.088901    0.8950     0.119255   \n",
       "22          1.0      0.680360        0.044448    0.9525     0.063096   \n",
       "32          2.0      0.692112        0.033365    0.9455     0.060561   \n",
       "0           2.0      0.560037        0.085395    0.8220     0.217350   \n",
       "18          2.0      0.673394        0.023616    0.9820     0.032235   \n",
       "26          2.0      0.545457        0.109179    0.8015     0.239234   \n",
       "28          2.0      0.572596        0.066132    0.7820     0.256967   \n",
       "17          1.0      0.648340        0.043398    0.9910     0.022655   \n",
       "\n",
       "    transfer    metric  \n",
       "23  0.386131  0.117490  \n",
       "7   0.290224  0.105307  \n",
       "15  0.329153  0.094887  \n",
       "16  0.512630  0.089700  \n",
       "19  0.567055  0.079669  \n",
       "2   0.467620  0.079255  \n",
       "6   0.529006  0.073981  \n",
       "35  0.525362  0.073847  \n",
       "29  0.437325  0.066786  \n",
       "31  0.385464  0.062704  \n",
       "27  0.576502  0.062137  \n",
       "30  0.405602  0.057891  \n",
       "1   0.596000  0.056141  \n",
       "33  0.469503  0.056038  \n",
       "10  0.536663  0.055273  \n",
       "14  0.492974  0.054377  \n",
       "34  0.343276  0.049903  \n",
       "8   0.598224  0.049372  \n",
       "13  0.372186  0.048270  \n",
       "5   0.404635  0.048263  \n",
       "12  0.541114  0.048248  \n",
       "9   0.604176  0.048013  \n",
       "4   0.485926  0.047231  \n",
       "24  0.430379  0.043300  \n",
       "3   0.599809  0.043272  \n",
       "25  0.444304  0.042585  \n",
       "20  0.569648  0.041033  \n",
       "21  0.431036  0.041017  \n",
       "11  0.610126  0.039329  \n",
       "22  0.406306  0.038784  \n",
       "32  0.470918  0.034579  \n",
       "0   0.685548  0.030611  \n",
       "18  0.233880  0.026880  \n",
       "26  0.684902  0.026173  \n",
       "28  0.677747  0.021976  \n",
       "17  0.227202  0.020697  "
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from pathlib import Path\n",
    "import shutil\n",
    "results_dir = 'results_grid'\n",
    "runs = []\n",
    "\n",
    "for filename in os.listdir(results_dir):\n",
    "    filepath = Path(results_dir)/Path(filename)\n",
    "    if filename.endswith('.pkl'):\n",
    "        runs.append(torch.load(filepath))\n",
    "        \n",
    "runs = [run for run in runs if (run['args']['dataset_size']==125)]\n",
    "        \n",
    "data = {}\n",
    "for arg_name in runs[0]['args'].keys():\n",
    "    data[arg_name] = [run['args'][arg_name] for run in runs]\n",
    "data['pickle']   = np.array([run['args']['pickle'].split('/')[-1].rstrip('.pkl') for run in runs])\n",
    "    \n",
    "data['bleu_quality']   = np.array([run['delexicalised_metrics']['bleu_scores']['quality']['avg'] for run in runs])\n",
    "data['bleu_diversity'] = np.array([run['delexicalised_metrics']['bleu_scores']['diversity']['avg'] for run in runs])\n",
    "data['accuracy']       = np.array([run['delexicalised_metrics']['intent_accuracy'] for run in runs])\n",
    "data['originality']       = np.array([run['delexicalised_metrics']['originality']['avg'] for run in runs])\n",
    "data['transfer']   = np.array([run['delexicalised_metrics']['transfer']['avg'] for run in runs])\n",
    "\n",
    "data['metric'] = data['accuracy']**10 * data['originality'] #* data['bleu_quality']\n",
    "\n",
    "df = pd.DataFrame(data, columns=['pickle', 'dataset_size', 'n_epochs', 'seed', #'x1', 'x2', 'k1', 'k2', \n",
    "                                 'hidden_size', 'embedding_dimension', 'temperature',\n",
    "                                 'bleu_quality', 'bleu_diversity', 'accuracy', 'originality', 'transfer', 'metric'])\n",
    "df = df.sort_values(by = 'metric', ascending=False)\n",
    "print(len(runs))\n",
    "df.head(100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Grid search 2 : encoder vs decoder"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "python -m grid results_grid_2 \"python automatic_data_generation/train_and_eval_cvae.py -ng 2000 -ep 50 --temperature 0.5\" --dataset-size:int 125 250 500 1000 --seed:int 1 2 3 --temperature:float 0.5 --embedding-dimension:int 100 300 --hidden-size-encoder 256 512 --hidden-size-decoder 256 512 --num-layers-encoder 1 2 --num-layers-decoder 1 2 --n_parallel 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "48\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pickle</th>\n",
       "      <th>dataset_size</th>\n",
       "      <th>n_epochs</th>\n",
       "      <th>seed</th>\n",
       "      <th>embedding_dimension</th>\n",
       "      <th>hidden_size_encoder</th>\n",
       "      <th>hidden_size_decoder</th>\n",
       "      <th>num_layers_encoder</th>\n",
       "      <th>num_layers_decoder</th>\n",
       "      <th>bleu_quality</th>\n",
       "      <th>bleu_diversity</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>originality</th>\n",
       "      <th>transfer</th>\n",
       "      <th>metric</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>15832</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>100</td>\n",
       "      <td>512</td>\n",
       "      <td>256</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.611668</td>\n",
       "      <td>0.097473</td>\n",
       "      <td>0.974891</td>\n",
       "      <td>0.487479</td>\n",
       "      <td>0.235702</td>\n",
       "      <td>3.780215e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10861</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>3</td>\n",
       "      <td>100</td>\n",
       "      <td>256</td>\n",
       "      <td>256</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.626343</td>\n",
       "      <td>0.109180</td>\n",
       "      <td>0.965018</td>\n",
       "      <td>0.521307</td>\n",
       "      <td>0.253679</td>\n",
       "      <td>3.651299e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>16756</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>100</td>\n",
       "      <td>256</td>\n",
       "      <td>256</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.654596</td>\n",
       "      <td>0.094833</td>\n",
       "      <td>0.972463</td>\n",
       "      <td>0.459773</td>\n",
       "      <td>0.203465</td>\n",
       "      <td>3.477562e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>31140</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>3</td>\n",
       "      <td>100</td>\n",
       "      <td>512</td>\n",
       "      <td>256</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.590995</td>\n",
       "      <td>0.127982</td>\n",
       "      <td>0.966964</td>\n",
       "      <td>0.473487</td>\n",
       "      <td>0.225481</td>\n",
       "      <td>3.383837e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>12051</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>100</td>\n",
       "      <td>256</td>\n",
       "      <td>256</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.664477</td>\n",
       "      <td>0.082993</td>\n",
       "      <td>0.965074</td>\n",
       "      <td>0.472456</td>\n",
       "      <td>0.272588</td>\n",
       "      <td>3.311073e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>21959</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>100</td>\n",
       "      <td>512</td>\n",
       "      <td>256</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.647282</td>\n",
       "      <td>0.102840</td>\n",
       "      <td>0.955318</td>\n",
       "      <td>0.442412</td>\n",
       "      <td>0.285821</td>\n",
       "      <td>2.800970e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>06875</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>300</td>\n",
       "      <td>256</td>\n",
       "      <td>256</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.686757</td>\n",
       "      <td>0.045655</td>\n",
       "      <td>0.989671</td>\n",
       "      <td>0.253479</td>\n",
       "      <td>0.117370</td>\n",
       "      <td>2.284814e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>08722</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>3</td>\n",
       "      <td>300</td>\n",
       "      <td>512</td>\n",
       "      <td>256</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.608650</td>\n",
       "      <td>0.101259</td>\n",
       "      <td>0.997937</td>\n",
       "      <td>0.217189</td>\n",
       "      <td>0.045993</td>\n",
       "      <td>2.127495e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>01654</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>300</td>\n",
       "      <td>512</td>\n",
       "      <td>256</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.660595</td>\n",
       "      <td>0.044978</td>\n",
       "      <td>0.991207</td>\n",
       "      <td>0.224995</td>\n",
       "      <td>0.108662</td>\n",
       "      <td>2.059758e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>05967</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>3</td>\n",
       "      <td>300</td>\n",
       "      <td>256</td>\n",
       "      <td>256</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.607678</td>\n",
       "      <td>0.065273</td>\n",
       "      <td>0.983011</td>\n",
       "      <td>0.238574</td>\n",
       "      <td>0.085404</td>\n",
       "      <td>2.010054e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>17539</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>300</td>\n",
       "      <td>512</td>\n",
       "      <td>256</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.678007</td>\n",
       "      <td>0.044041</td>\n",
       "      <td>0.995002</td>\n",
       "      <td>0.210133</td>\n",
       "      <td>0.050600</td>\n",
       "      <td>1.998646e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>32000</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>300</td>\n",
       "      <td>256</td>\n",
       "      <td>256</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.648619</td>\n",
       "      <td>0.048849</td>\n",
       "      <td>0.977554</td>\n",
       "      <td>0.208008</td>\n",
       "      <td>0.182004</td>\n",
       "      <td>1.657625e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>08948</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>300</td>\n",
       "      <td>256</td>\n",
       "      <td>512</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.362500</td>\n",
       "      <td>0.236198</td>\n",
       "      <td>0.591235</td>\n",
       "      <td>0.781996</td>\n",
       "      <td>0.627093</td>\n",
       "      <td>4.081372e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>20266</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>300</td>\n",
       "      <td>256</td>\n",
       "      <td>512</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.310639</td>\n",
       "      <td>0.195389</td>\n",
       "      <td>0.506958</td>\n",
       "      <td>0.857272</td>\n",
       "      <td>0.615899</td>\n",
       "      <td>9.612497e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>24652</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>300</td>\n",
       "      <td>256</td>\n",
       "      <td>256</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0.392799</td>\n",
       "      <td>0.048135</td>\n",
       "      <td>0.520044</td>\n",
       "      <td>0.573311</td>\n",
       "      <td>0.522824</td>\n",
       "      <td>8.294538e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>39051</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>100</td>\n",
       "      <td>256</td>\n",
       "      <td>512</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.303231</td>\n",
       "      <td>0.268627</td>\n",
       "      <td>0.486694</td>\n",
       "      <td>0.859052</td>\n",
       "      <td>0.571974</td>\n",
       "      <td>6.405857e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>28457</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>3</td>\n",
       "      <td>300</td>\n",
       "      <td>256</td>\n",
       "      <td>512</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.321664</td>\n",
       "      <td>0.346537</td>\n",
       "      <td>0.453553</td>\n",
       "      <td>0.818073</td>\n",
       "      <td>0.603303</td>\n",
       "      <td>3.013484e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>17843</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>100</td>\n",
       "      <td>256</td>\n",
       "      <td>512</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.206993</td>\n",
       "      <td>0.190273</td>\n",
       "      <td>0.424096</td>\n",
       "      <td>0.927299</td>\n",
       "      <td>0.587842</td>\n",
       "      <td>1.745257e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>37368</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>300</td>\n",
       "      <td>512</td>\n",
       "      <td>256</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0.357960</td>\n",
       "      <td>0.036281</td>\n",
       "      <td>0.430414</td>\n",
       "      <td>0.580106</td>\n",
       "      <td>0.524730</td>\n",
       "      <td>1.265817e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>07668</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>3</td>\n",
       "      <td>300</td>\n",
       "      <td>256</td>\n",
       "      <td>256</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0.247891</td>\n",
       "      <td>0.164042</td>\n",
       "      <td>0.389126</td>\n",
       "      <td>0.716667</td>\n",
       "      <td>0.531158</td>\n",
       "      <td>5.704595e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>48409</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>3</td>\n",
       "      <td>100</td>\n",
       "      <td>512</td>\n",
       "      <td>256</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0.305124</td>\n",
       "      <td>0.129643</td>\n",
       "      <td>0.378893</td>\n",
       "      <td>0.739657</td>\n",
       "      <td>0.640482</td>\n",
       "      <td>4.510250e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>28218</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>3</td>\n",
       "      <td>100</td>\n",
       "      <td>512</td>\n",
       "      <td>512</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.189248</td>\n",
       "      <td>0.238243</td>\n",
       "      <td>0.346794</td>\n",
       "      <td>0.949236</td>\n",
       "      <td>0.658311</td>\n",
       "      <td>2.388278e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>17455</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>300</td>\n",
       "      <td>512</td>\n",
       "      <td>512</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.254907</td>\n",
       "      <td>0.246041</td>\n",
       "      <td>0.343323</td>\n",
       "      <td>0.880724</td>\n",
       "      <td>0.637634</td>\n",
       "      <td>2.003865e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>43596</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>300</td>\n",
       "      <td>512</td>\n",
       "      <td>512</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.255477</td>\n",
       "      <td>0.201274</td>\n",
       "      <td>0.343538</td>\n",
       "      <td>0.874367</td>\n",
       "      <td>0.613058</td>\n",
       "      <td>2.001885e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>13640</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>100</td>\n",
       "      <td>256</td>\n",
       "      <td>256</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0.280895</td>\n",
       "      <td>0.156792</td>\n",
       "      <td>0.342043</td>\n",
       "      <td>0.770646</td>\n",
       "      <td>0.638710</td>\n",
       "      <td>1.689141e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>33104</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>3</td>\n",
       "      <td>100</td>\n",
       "      <td>256</td>\n",
       "      <td>512</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.216772</td>\n",
       "      <td>0.334639</td>\n",
       "      <td>0.311602</td>\n",
       "      <td>0.919346</td>\n",
       "      <td>0.652941</td>\n",
       "      <td>7.933716e-06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>39206</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>100</td>\n",
       "      <td>512</td>\n",
       "      <td>256</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0.260658</td>\n",
       "      <td>0.132655</td>\n",
       "      <td>0.311167</td>\n",
       "      <td>0.844622</td>\n",
       "      <td>0.600721</td>\n",
       "      <td>7.187786e-06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>24276</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>3</td>\n",
       "      <td>100</td>\n",
       "      <td>256</td>\n",
       "      <td>256</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0.233695</td>\n",
       "      <td>0.123888</td>\n",
       "      <td>0.306227</td>\n",
       "      <td>0.816301</td>\n",
       "      <td>0.611024</td>\n",
       "      <td>5.919577e-06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>08115</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>100</td>\n",
       "      <td>512</td>\n",
       "      <td>512</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.154916</td>\n",
       "      <td>0.296606</td>\n",
       "      <td>0.291145</td>\n",
       "      <td>0.974133</td>\n",
       "      <td>0.639227</td>\n",
       "      <td>4.263002e-06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>49486</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>300</td>\n",
       "      <td>256</td>\n",
       "      <td>256</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0.180171</td>\n",
       "      <td>0.062394</td>\n",
       "      <td>0.278442</td>\n",
       "      <td>0.779934</td>\n",
       "      <td>0.559230</td>\n",
       "      <td>2.184753e-06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>25957</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>3</td>\n",
       "      <td>300</td>\n",
       "      <td>512</td>\n",
       "      <td>512</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.194661</td>\n",
       "      <td>0.153748</td>\n",
       "      <td>0.272075</td>\n",
       "      <td>0.943588</td>\n",
       "      <td>0.573001</td>\n",
       "      <td>2.097310e-06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>36472</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>100</td>\n",
       "      <td>512</td>\n",
       "      <td>512</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.166842</td>\n",
       "      <td>0.268466</td>\n",
       "      <td>0.259285</td>\n",
       "      <td>0.953399</td>\n",
       "      <td>0.689097</td>\n",
       "      <td>1.309313e-06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>32716</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>3</td>\n",
       "      <td>300</td>\n",
       "      <td>512</td>\n",
       "      <td>256</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0.193839</td>\n",
       "      <td>0.073227</td>\n",
       "      <td>0.249465</td>\n",
       "      <td>0.778729</td>\n",
       "      <td>0.590407</td>\n",
       "      <td>7.269184e-07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>26500</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>300</td>\n",
       "      <td>512</td>\n",
       "      <td>256</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0.194328</td>\n",
       "      <td>0.043654</td>\n",
       "      <td>0.242641</td>\n",
       "      <td>0.777848</td>\n",
       "      <td>0.563592</td>\n",
       "      <td>5.502297e-07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>33318</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>100</td>\n",
       "      <td>512</td>\n",
       "      <td>256</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0.173277</td>\n",
       "      <td>0.090939</td>\n",
       "      <td>0.224182</td>\n",
       "      <td>0.886665</td>\n",
       "      <td>0.661931</td>\n",
       "      <td>2.842903e-07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>34904</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>300</td>\n",
       "      <td>256</td>\n",
       "      <td>512</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0.136894</td>\n",
       "      <td>0.372094</td>\n",
       "      <td>0.187450</td>\n",
       "      <td>0.958068</td>\n",
       "      <td>0.683963</td>\n",
       "      <td>5.131496e-08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>36143</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>100</td>\n",
       "      <td>512</td>\n",
       "      <td>512</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0.103202</td>\n",
       "      <td>0.337376</td>\n",
       "      <td>0.168375</td>\n",
       "      <td>0.984249</td>\n",
       "      <td>0.673854</td>\n",
       "      <td>1.802549e-08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>46641</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>100</td>\n",
       "      <td>256</td>\n",
       "      <td>512</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0.099115</td>\n",
       "      <td>0.417362</td>\n",
       "      <td>0.166409</td>\n",
       "      <td>0.990183</td>\n",
       "      <td>0.703271</td>\n",
       "      <td>1.612481e-08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>31159</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>3</td>\n",
       "      <td>300</td>\n",
       "      <td>256</td>\n",
       "      <td>512</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0.116824</td>\n",
       "      <td>0.379021</td>\n",
       "      <td>0.162978</td>\n",
       "      <td>0.967497</td>\n",
       "      <td>0.686444</td>\n",
       "      <td>1.279166e-08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>42477</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>3</td>\n",
       "      <td>100</td>\n",
       "      <td>512</td>\n",
       "      <td>512</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0.109252</td>\n",
       "      <td>0.211424</td>\n",
       "      <td>0.156619</td>\n",
       "      <td>0.987091</td>\n",
       "      <td>0.688028</td>\n",
       "      <td>8.766097e-09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>07182</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>3</td>\n",
       "      <td>100</td>\n",
       "      <td>256</td>\n",
       "      <td>512</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0.091053</td>\n",
       "      <td>0.428506</td>\n",
       "      <td>0.149505</td>\n",
       "      <td>0.988360</td>\n",
       "      <td>0.714039</td>\n",
       "      <td>5.514026e-09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>10453</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>300</td>\n",
       "      <td>256</td>\n",
       "      <td>512</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0.112392</td>\n",
       "      <td>0.318400</td>\n",
       "      <td>0.147984</td>\n",
       "      <td>0.971403</td>\n",
       "      <td>0.695877</td>\n",
       "      <td>4.892519e-09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>08562</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>3</td>\n",
       "      <td>300</td>\n",
       "      <td>512</td>\n",
       "      <td>512</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0.079305</td>\n",
       "      <td>0.480470</td>\n",
       "      <td>0.145633</td>\n",
       "      <td>0.985882</td>\n",
       "      <td>0.720852</td>\n",
       "      <td>4.230821e-09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>16501</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>100</td>\n",
       "      <td>256</td>\n",
       "      <td>512</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0.106390</td>\n",
       "      <td>0.263795</td>\n",
       "      <td>0.143668</td>\n",
       "      <td>0.987376</td>\n",
       "      <td>0.676262</td>\n",
       "      <td>3.699087e-09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>45362</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>100</td>\n",
       "      <td>512</td>\n",
       "      <td>512</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0.056672</td>\n",
       "      <td>0.564440</td>\n",
       "      <td>0.139739</td>\n",
       "      <td>0.996864</td>\n",
       "      <td>0.718615</td>\n",
       "      <td>2.830240e-09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>44980</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>300</td>\n",
       "      <td>512</td>\n",
       "      <td>512</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0.091936</td>\n",
       "      <td>0.428139</td>\n",
       "      <td>0.130958</td>\n",
       "      <td>0.989438</td>\n",
       "      <td>0.714321</td>\n",
       "      <td>1.467974e-09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>33281</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>300</td>\n",
       "      <td>512</td>\n",
       "      <td>512</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0.114789</td>\n",
       "      <td>0.255089</td>\n",
       "      <td>0.129904</td>\n",
       "      <td>0.978251</td>\n",
       "      <td>0.683618</td>\n",
       "      <td>1.338710e-09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>35755</td>\n",
       "      <td>500</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>100</td>\n",
       "      <td>256</td>\n",
       "      <td>256</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0.097395</td>\n",
       "      <td>0.418140</td>\n",
       "      <td>0.123093</td>\n",
       "      <td>0.985551</td>\n",
       "      <td>0.688732</td>\n",
       "      <td>7.870499e-10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   pickle  dataset_size  n_epochs  seed  embedding_dimension  \\\n",
       "2   15832  500           50         1    100                   \n",
       "0   10861  500           50         3    100                   \n",
       "41  16756  500           50         2    100                   \n",
       "32  31140  500           50         3    100                   \n",
       "30  12051  500           50         1    100                   \n",
       "25  21959  500           50         2    100                   \n",
       "17  06875  500           50         1    300                   \n",
       "5   08722  500           50         3    300                   \n",
       "29  01654  500           50         2    300                   \n",
       "35  05967  500           50         3    300                   \n",
       "44  17539  500           50         1    300                   \n",
       "16  32000  500           50         2    300                   \n",
       "24  08948  500           50         1    300                   \n",
       "6   20266  500           50         2    300                   \n",
       "22  24652  500           50         1    300                   \n",
       "4   39051  500           50         2    100                   \n",
       "23  28457  500           50         3    300                   \n",
       "36  17843  500           50         1    100                   \n",
       "7   37368  500           50         1    300                   \n",
       "28  07668  500           50         3    300                   \n",
       "46  48409  500           50         3    100                   \n",
       "40  28218  500           50         3    100                   \n",
       "10  17455  500           50         1    300                   \n",
       "31  43596  500           50         2    300                   \n",
       "47  13640  500           50         1    100                   \n",
       "27  33104  500           50         3    100                   \n",
       "26  39206  500           50         1    100                   \n",
       "42  24276  500           50         3    100                   \n",
       "11  08115  500           50         2    100                   \n",
       "1   49486  500           50         2    300                   \n",
       "45  25957  500           50         3    300                   \n",
       "34  36472  500           50         1    100                   \n",
       "15  32716  500           50         3    300                   \n",
       "39  26500  500           50         2    300                   \n",
       "37  33318  500           50         2    100                   \n",
       "38  34904  500           50         1    300                   \n",
       "13  36143  500           50         2    100                   \n",
       "12  46641  500           50         2    100                   \n",
       "21  31159  500           50         3    300                   \n",
       "20  42477  500           50         3    100                   \n",
       "3   07182  500           50         3    100                   \n",
       "14  10453  500           50         2    300                   \n",
       "18  08562  500           50         3    300                   \n",
       "43  16501  500           50         1    100                   \n",
       "9   45362  500           50         1    100                   \n",
       "19  44980  500           50         1    300                   \n",
       "33  33281  500           50         2    300                   \n",
       "8   35755  500           50         2    100                   \n",
       "\n",
       "    hidden_size_encoder  hidden_size_decoder  num_layers_encoder  \\\n",
       "2   512                  256                   1                   \n",
       "0   256                  256                   1                   \n",
       "41  256                  256                   1                   \n",
       "32  512                  256                   1                   \n",
       "30  256                  256                   1                   \n",
       "25  512                  256                   1                   \n",
       "17  256                  256                   1                   \n",
       "5   512                  256                   1                   \n",
       "29  512                  256                   1                   \n",
       "35  256                  256                   1                   \n",
       "44  512                  256                   1                   \n",
       "16  256                  256                   1                   \n",
       "24  256                  512                   1                   \n",
       "6   256                  512                   1                   \n",
       "22  256                  256                   2                   \n",
       "4   256                  512                   1                   \n",
       "23  256                  512                   1                   \n",
       "36  256                  512                   1                   \n",
       "7   512                  256                   2                   \n",
       "28  256                  256                   2                   \n",
       "46  512                  256                   2                   \n",
       "40  512                  512                   1                   \n",
       "10  512                  512                   1                   \n",
       "31  512                  512                   1                   \n",
       "47  256                  256                   2                   \n",
       "27  256                  512                   1                   \n",
       "26  512                  256                   2                   \n",
       "42  256                  256                   2                   \n",
       "11  512                  512                   1                   \n",
       "1   256                  256                   2                   \n",
       "45  512                  512                   1                   \n",
       "34  512                  512                   1                   \n",
       "15  512                  256                   2                   \n",
       "39  512                  256                   2                   \n",
       "37  512                  256                   2                   \n",
       "38  256                  512                   2                   \n",
       "13  512                  512                   2                   \n",
       "12  256                  512                   2                   \n",
       "21  256                  512                   2                   \n",
       "20  512                  512                   2                   \n",
       "3   256                  512                   2                   \n",
       "14  256                  512                   2                   \n",
       "18  512                  512                   2                   \n",
       "43  256                  512                   2                   \n",
       "9   512                  512                   2                   \n",
       "19  512                  512                   2                   \n",
       "33  512                  512                   2                   \n",
       "8   256                  256                   2                   \n",
       "\n",
       "    num_layers_decoder  bleu_quality  bleu_diversity  accuracy  originality  \\\n",
       "2    1                  0.611668      0.097473        0.974891  0.487479      \n",
       "0    1                  0.626343      0.109180        0.965018  0.521307      \n",
       "41   1                  0.654596      0.094833        0.972463  0.459773      \n",
       "32   1                  0.590995      0.127982        0.966964  0.473487      \n",
       "30   1                  0.664477      0.082993        0.965074  0.472456      \n",
       "25   1                  0.647282      0.102840        0.955318  0.442412      \n",
       "17   1                  0.686757      0.045655        0.989671  0.253479      \n",
       "5    1                  0.608650      0.101259        0.997937  0.217189      \n",
       "29   1                  0.660595      0.044978        0.991207  0.224995      \n",
       "35   1                  0.607678      0.065273        0.983011  0.238574      \n",
       "44   1                  0.678007      0.044041        0.995002  0.210133      \n",
       "16   1                  0.648619      0.048849        0.977554  0.208008      \n",
       "24   1                  0.362500      0.236198        0.591235  0.781996      \n",
       "6    1                  0.310639      0.195389        0.506958  0.857272      \n",
       "22   1                  0.392799      0.048135        0.520044  0.573311      \n",
       "4    1                  0.303231      0.268627        0.486694  0.859052      \n",
       "23   1                  0.321664      0.346537        0.453553  0.818073      \n",
       "36   1                  0.206993      0.190273        0.424096  0.927299      \n",
       "7    1                  0.357960      0.036281        0.430414  0.580106      \n",
       "28   1                  0.247891      0.164042        0.389126  0.716667      \n",
       "46   1                  0.305124      0.129643        0.378893  0.739657      \n",
       "40   1                  0.189248      0.238243        0.346794  0.949236      \n",
       "10   1                  0.254907      0.246041        0.343323  0.880724      \n",
       "31   1                  0.255477      0.201274        0.343538  0.874367      \n",
       "47   1                  0.280895      0.156792        0.342043  0.770646      \n",
       "27   1                  0.216772      0.334639        0.311602  0.919346      \n",
       "26   1                  0.260658      0.132655        0.311167  0.844622      \n",
       "42   1                  0.233695      0.123888        0.306227  0.816301      \n",
       "11   1                  0.154916      0.296606        0.291145  0.974133      \n",
       "1    1                  0.180171      0.062394        0.278442  0.779934      \n",
       "45   1                  0.194661      0.153748        0.272075  0.943588      \n",
       "34   1                  0.166842      0.268466        0.259285  0.953399      \n",
       "15   1                  0.193839      0.073227        0.249465  0.778729      \n",
       "39   1                  0.194328      0.043654        0.242641  0.777848      \n",
       "37   1                  0.173277      0.090939        0.224182  0.886665      \n",
       "38   1                  0.136894      0.372094        0.187450  0.958068      \n",
       "13   1                  0.103202      0.337376        0.168375  0.984249      \n",
       "12   1                  0.099115      0.417362        0.166409  0.990183      \n",
       "21   1                  0.116824      0.379021        0.162978  0.967497      \n",
       "20   1                  0.109252      0.211424        0.156619  0.987091      \n",
       "3    1                  0.091053      0.428506        0.149505  0.988360      \n",
       "14   1                  0.112392      0.318400        0.147984  0.971403      \n",
       "18   1                  0.079305      0.480470        0.145633  0.985882      \n",
       "43   1                  0.106390      0.263795        0.143668  0.987376      \n",
       "9    1                  0.056672      0.564440        0.139739  0.996864      \n",
       "19   1                  0.091936      0.428139        0.130958  0.989438      \n",
       "33   1                  0.114789      0.255089        0.129904  0.978251      \n",
       "8    1                  0.097395      0.418140        0.123093  0.985551      \n",
       "\n",
       "    transfer        metric  \n",
       "2   0.235702  3.780215e-01  \n",
       "0   0.253679  3.651299e-01  \n",
       "41  0.203465  3.477562e-01  \n",
       "32  0.225481  3.383837e-01  \n",
       "30  0.272588  3.311073e-01  \n",
       "25  0.285821  2.800970e-01  \n",
       "17  0.117370  2.284814e-01  \n",
       "5   0.045993  2.127495e-01  \n",
       "29  0.108662  2.059758e-01  \n",
       "35  0.085404  2.010054e-01  \n",
       "44  0.050600  1.998646e-01  \n",
       "16  0.182004  1.657625e-01  \n",
       "24  0.627093  4.081372e-03  \n",
       "6   0.615899  9.612497e-04  \n",
       "22  0.522824  8.294538e-04  \n",
       "4   0.571974  6.405857e-04  \n",
       "23  0.603303  3.013484e-04  \n",
       "36  0.587842  1.745257e-04  \n",
       "7   0.524730  1.265817e-04  \n",
       "28  0.531158  5.704595e-05  \n",
       "46  0.640482  4.510250e-05  \n",
       "40  0.658311  2.388278e-05  \n",
       "10  0.637634  2.003865e-05  \n",
       "31  0.613058  2.001885e-05  \n",
       "47  0.638710  1.689141e-05  \n",
       "27  0.652941  7.933716e-06  \n",
       "26  0.600721  7.187786e-06  \n",
       "42  0.611024  5.919577e-06  \n",
       "11  0.639227  4.263002e-06  \n",
       "1   0.559230  2.184753e-06  \n",
       "45  0.573001  2.097310e-06  \n",
       "34  0.689097  1.309313e-06  \n",
       "15  0.590407  7.269184e-07  \n",
       "39  0.563592  5.502297e-07  \n",
       "37  0.661931  2.842903e-07  \n",
       "38  0.683963  5.131496e-08  \n",
       "13  0.673854  1.802549e-08  \n",
       "12  0.703271  1.612481e-08  \n",
       "21  0.686444  1.279166e-08  \n",
       "20  0.688028  8.766097e-09  \n",
       "3   0.714039  5.514026e-09  \n",
       "14  0.695877  4.892519e-09  \n",
       "18  0.720852  4.230821e-09  \n",
       "43  0.676262  3.699087e-09  \n",
       "9   0.718615  2.830240e-09  \n",
       "19  0.714321  1.467974e-09  \n",
       "33  0.683618  1.338710e-09  \n",
       "8   0.688732  7.870499e-10  "
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from pathlib import Path\n",
    "import shutil\n",
    "results_dir = 'results_grid_2'\n",
    "runs = []\n",
    "\n",
    "for filename in os.listdir(results_dir):\n",
    "    filepath = Path(results_dir)/Path(filename)\n",
    "    if filename.endswith('.pkl'):\n",
    "        runs.append(torch.load(filepath))\n",
    "        \n",
    "runs = [run for run in runs if (run['args']['dataset_size']==500)]\n",
    "        \n",
    "data = {}\n",
    "for arg_name in runs[0]['args'].keys():\n",
    "    data[arg_name] = [run['args'][arg_name] for run in runs]\n",
    "data['pickle']   = np.array([run['args']['pickle'].split('/')[-1].rstrip('.pkl') for run in runs])\n",
    "    \n",
    "data['bleu_quality']   = np.array([run['delexicalised_metrics']['bleu_scores']['quality']['avg'] for run in runs])\n",
    "data['bleu_diversity'] = np.array([run['delexicalised_metrics']['bleu_scores']['diversity']['avg'] for run in runs])\n",
    "data['accuracy']       = np.array([run['delexicalised_metrics']['intent_accuracy']['avg'] for run in runs])\n",
    "data['originality']       = np.array([run['delexicalised_metrics']['originality']['avg'] for run in runs])\n",
    "data['transfer']   = np.array([run['delexicalised_metrics']['transfer']['metric']['avg'] for run in runs])\n",
    "\n",
    "data['metric'] = data['accuracy']**10 * data['originality'] #* data['bleu_quality']\n",
    "\n",
    "df = pd.DataFrame(data, columns=['pickle', 'dataset_size', 'n_epochs', 'seed', 'embedding_dimension',#'x1', 'x2', 'k1', 'k2', \n",
    "                                 'hidden_size_encoder', 'hidden_size_decoder', 'num_layers_encoder', 'num_layers_decoder',\n",
    "                                 'bleu_quality', 'bleu_diversity', 'accuracy', 'originality', 'transfer', 'metric'])\n",
    "df = df.sort_values(by = 'metric', ascending=False)\n",
    "print(len(runs))\n",
    "df.head(1000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Grid search 3 : transfer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "python -m grid results_grid_3 \"python automatic_data_generation/train_and_eval_cvae.py -ng 2000 -ep 50 --none-type snips-dump\" --dataset-size:int 125 250 500 1000 --seed:int 1 2 3 --cosine-threshold:float 0 0.8 0.9 0.95 --none-size:int 125 250 500 1000 --n_parallel 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pickle</th>\n",
       "      <th>dataset_size</th>\n",
       "      <th>n_epochs</th>\n",
       "      <th>seed</th>\n",
       "      <th>embedding_dimension</th>\n",
       "      <th>none_size</th>\n",
       "      <th>cosine_threshold</th>\n",
       "      <th>bleu_quality</th>\n",
       "      <th>bleu_diversity</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>originality</th>\n",
       "      <th>transfer</th>\n",
       "      <th>metric</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>17784</td>\n",
       "      <td>1000</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>100</td>\n",
       "      <td>250</td>\n",
       "      <td>0.95</td>\n",
       "      <td>0.393728</td>\n",
       "      <td>0.237461</td>\n",
       "      <td>0.717383</td>\n",
       "      <td>0.838710</td>\n",
       "      <td>0.493467</td>\n",
       "      <td>3.610037e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>43621</td>\n",
       "      <td>1000</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>100</td>\n",
       "      <td>500</td>\n",
       "      <td>0.95</td>\n",
       "      <td>0.432848</td>\n",
       "      <td>0.248355</td>\n",
       "      <td>0.645143</td>\n",
       "      <td>0.773662</td>\n",
       "      <td>0.298874</td>\n",
       "      <td>1.248998e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>13437</td>\n",
       "      <td>1000</td>\n",
       "      <td>50</td>\n",
       "      <td>3</td>\n",
       "      <td>100</td>\n",
       "      <td>250</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.333354</td>\n",
       "      <td>0.215302</td>\n",
       "      <td>0.639930</td>\n",
       "      <td>0.887567</td>\n",
       "      <td>0.539501</td>\n",
       "      <td>1.151660e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>47717</td>\n",
       "      <td>1000</td>\n",
       "      <td>50</td>\n",
       "      <td>3</td>\n",
       "      <td>100</td>\n",
       "      <td>125</td>\n",
       "      <td>0.95</td>\n",
       "      <td>0.435449</td>\n",
       "      <td>0.245025</td>\n",
       "      <td>0.580291</td>\n",
       "      <td>0.813852</td>\n",
       "      <td>0.424630</td>\n",
       "      <td>4.329743e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>20445</td>\n",
       "      <td>1000</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>100</td>\n",
       "      <td>125</td>\n",
       "      <td>0.95</td>\n",
       "      <td>0.421165</td>\n",
       "      <td>0.316455</td>\n",
       "      <td>0.527633</td>\n",
       "      <td>0.780232</td>\n",
       "      <td>0.431380</td>\n",
       "      <td>1.672334e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>20773</td>\n",
       "      <td>1000</td>\n",
       "      <td>50</td>\n",
       "      <td>3</td>\n",
       "      <td>100</td>\n",
       "      <td>500</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.367772</td>\n",
       "      <td>0.461277</td>\n",
       "      <td>0.503981</td>\n",
       "      <td>0.804166</td>\n",
       "      <td>0.592593</td>\n",
       "      <td>1.057165e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>44204</td>\n",
       "      <td>1000</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>100</td>\n",
       "      <td>125</td>\n",
       "      <td>0.90</td>\n",
       "      <td>0.478195</td>\n",
       "      <td>0.309747</td>\n",
       "      <td>0.498144</td>\n",
       "      <td>0.792718</td>\n",
       "      <td>0.464515</td>\n",
       "      <td>9.409081e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>38072</td>\n",
       "      <td>1000</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>100</td>\n",
       "      <td>500</td>\n",
       "      <td>0.80</td>\n",
       "      <td>0.194440</td>\n",
       "      <td>0.454097</td>\n",
       "      <td>0.494651</td>\n",
       "      <td>0.948659</td>\n",
       "      <td>0.583499</td>\n",
       "      <td>8.769759e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>23894</td>\n",
       "      <td>1000</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>100</td>\n",
       "      <td>250</td>\n",
       "      <td>0.80</td>\n",
       "      <td>0.395414</td>\n",
       "      <td>0.302146</td>\n",
       "      <td>0.489667</td>\n",
       "      <td>0.873030</td>\n",
       "      <td>0.509737</td>\n",
       "      <td>7.925198e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>32058</td>\n",
       "      <td>1000</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>100</td>\n",
       "      <td>125</td>\n",
       "      <td>0.80</td>\n",
       "      <td>0.381793</td>\n",
       "      <td>0.283795</td>\n",
       "      <td>0.488118</td>\n",
       "      <td>0.907601</td>\n",
       "      <td>0.473487</td>\n",
       "      <td>7.678044e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>06771</td>\n",
       "      <td>1000</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>100</td>\n",
       "      <td>125</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.255093</td>\n",
       "      <td>0.342376</td>\n",
       "      <td>0.479748</td>\n",
       "      <td>0.964264</td>\n",
       "      <td>0.589489</td>\n",
       "      <td>6.458545e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>09428</td>\n",
       "      <td>1000</td>\n",
       "      <td>50</td>\n",
       "      <td>3</td>\n",
       "      <td>100</td>\n",
       "      <td>250</td>\n",
       "      <td>0.95</td>\n",
       "      <td>0.334538</td>\n",
       "      <td>0.352572</td>\n",
       "      <td>0.478821</td>\n",
       "      <td>0.828161</td>\n",
       "      <td>0.480083</td>\n",
       "      <td>6.334806e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>28109</td>\n",
       "      <td>1000</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>100</td>\n",
       "      <td>250</td>\n",
       "      <td>0.90</td>\n",
       "      <td>0.343504</td>\n",
       "      <td>0.333271</td>\n",
       "      <td>0.468147</td>\n",
       "      <td>0.907099</td>\n",
       "      <td>0.464829</td>\n",
       "      <td>5.056167e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>26207</td>\n",
       "      <td>1000</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>100</td>\n",
       "      <td>1000</td>\n",
       "      <td>0.90</td>\n",
       "      <td>0.293979</td>\n",
       "      <td>0.453825</td>\n",
       "      <td>0.459267</td>\n",
       "      <td>0.901299</td>\n",
       "      <td>0.644951</td>\n",
       "      <td>4.174959e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>03532</td>\n",
       "      <td>1000</td>\n",
       "      <td>50</td>\n",
       "      <td>3</td>\n",
       "      <td>100</td>\n",
       "      <td>250</td>\n",
       "      <td>0.90</td>\n",
       "      <td>0.392849</td>\n",
       "      <td>0.426362</td>\n",
       "      <td>0.452816</td>\n",
       "      <td>0.822553</td>\n",
       "      <td>0.479514</td>\n",
       "      <td>3.624281e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>19224</td>\n",
       "      <td>1000</td>\n",
       "      <td>50</td>\n",
       "      <td>3</td>\n",
       "      <td>100</td>\n",
       "      <td>125</td>\n",
       "      <td>0.80</td>\n",
       "      <td>0.273148</td>\n",
       "      <td>0.502153</td>\n",
       "      <td>0.443929</td>\n",
       "      <td>0.912004</td>\n",
       "      <td>0.545504</td>\n",
       "      <td>2.972623e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>10391</td>\n",
       "      <td>1000</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>100</td>\n",
       "      <td>125</td>\n",
       "      <td>0.95</td>\n",
       "      <td>0.338473</td>\n",
       "      <td>0.346794</td>\n",
       "      <td>0.432122</td>\n",
       "      <td>0.921764</td>\n",
       "      <td>0.469238</td>\n",
       "      <td>2.270175e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>35230</td>\n",
       "      <td>1000</td>\n",
       "      <td>50</td>\n",
       "      <td>3</td>\n",
       "      <td>100</td>\n",
       "      <td>500</td>\n",
       "      <td>0.95</td>\n",
       "      <td>0.321875</td>\n",
       "      <td>0.418420</td>\n",
       "      <td>0.420737</td>\n",
       "      <td>0.817312</td>\n",
       "      <td>0.489134</td>\n",
       "      <td>1.738238e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>32204</td>\n",
       "      <td>1000</td>\n",
       "      <td>50</td>\n",
       "      <td>3</td>\n",
       "      <td>100</td>\n",
       "      <td>500</td>\n",
       "      <td>0.90</td>\n",
       "      <td>0.417755</td>\n",
       "      <td>0.333813</td>\n",
       "      <td>0.417422</td>\n",
       "      <td>0.866784</td>\n",
       "      <td>0.544611</td>\n",
       "      <td>1.606015e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>17684</td>\n",
       "      <td>1000</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>100</td>\n",
       "      <td>250</td>\n",
       "      <td>0.90</td>\n",
       "      <td>0.335847</td>\n",
       "      <td>0.370672</td>\n",
       "      <td>0.414701</td>\n",
       "      <td>0.927364</td>\n",
       "      <td>0.461646</td>\n",
       "      <td>1.504359e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>42380</td>\n",
       "      <td>1000</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>100</td>\n",
       "      <td>500</td>\n",
       "      <td>0.90</td>\n",
       "      <td>0.357201</td>\n",
       "      <td>0.414022</td>\n",
       "      <td>0.403791</td>\n",
       "      <td>0.809686</td>\n",
       "      <td>0.463653</td>\n",
       "      <td>1.152314e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>12842</td>\n",
       "      <td>1000</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>100</td>\n",
       "      <td>250</td>\n",
       "      <td>0.95</td>\n",
       "      <td>0.277537</td>\n",
       "      <td>0.528968</td>\n",
       "      <td>0.382628</td>\n",
       "      <td>0.917369</td>\n",
       "      <td>0.468548</td>\n",
       "      <td>6.726160e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>37436</td>\n",
       "      <td>1000</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>100</td>\n",
       "      <td>1000</td>\n",
       "      <td>0.95</td>\n",
       "      <td>0.310948</td>\n",
       "      <td>0.460927</td>\n",
       "      <td>0.380619</td>\n",
       "      <td>0.883294</td>\n",
       "      <td>0.542228</td>\n",
       "      <td>6.381312e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>02348</td>\n",
       "      <td>1000</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>100</td>\n",
       "      <td>125</td>\n",
       "      <td>0.80</td>\n",
       "      <td>0.388615</td>\n",
       "      <td>0.410206</td>\n",
       "      <td>0.379967</td>\n",
       "      <td>0.853646</td>\n",
       "      <td>0.409726</td>\n",
       "      <td>6.272751e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>07207</td>\n",
       "      <td>1000</td>\n",
       "      <td>50</td>\n",
       "      <td>3</td>\n",
       "      <td>100</td>\n",
       "      <td>125</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.369053</td>\n",
       "      <td>0.395023</td>\n",
       "      <td>0.379663</td>\n",
       "      <td>0.830798</td>\n",
       "      <td>0.490608</td>\n",
       "      <td>6.222711e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>43896</td>\n",
       "      <td>1000</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>100</td>\n",
       "      <td>1000</td>\n",
       "      <td>0.95</td>\n",
       "      <td>0.294404</td>\n",
       "      <td>0.300528</td>\n",
       "      <td>0.357913</td>\n",
       "      <td>0.900580</td>\n",
       "      <td>0.543948</td>\n",
       "      <td>3.449640e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>44530</td>\n",
       "      <td>1000</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>100</td>\n",
       "      <td>500</td>\n",
       "      <td>0.95</td>\n",
       "      <td>0.228726</td>\n",
       "      <td>0.435705</td>\n",
       "      <td>0.356450</td>\n",
       "      <td>0.852910</td>\n",
       "      <td>0.515948</td>\n",
       "      <td>3.311180e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>02708</td>\n",
       "      <td>1000</td>\n",
       "      <td>50</td>\n",
       "      <td>3</td>\n",
       "      <td>100</td>\n",
       "      <td>1000</td>\n",
       "      <td>0.95</td>\n",
       "      <td>0.276848</td>\n",
       "      <td>0.499742</td>\n",
       "      <td>0.355318</td>\n",
       "      <td>0.820321</td>\n",
       "      <td>0.493750</td>\n",
       "      <td>3.207585e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>12484</td>\n",
       "      <td>1000</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>100</td>\n",
       "      <td>1000</td>\n",
       "      <td>0.90</td>\n",
       "      <td>0.182331</td>\n",
       "      <td>0.479930</td>\n",
       "      <td>0.348470</td>\n",
       "      <td>0.943802</td>\n",
       "      <td>0.654712</td>\n",
       "      <td>2.640272e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>39873</td>\n",
       "      <td>1000</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>100</td>\n",
       "      <td>250</td>\n",
       "      <td>0.80</td>\n",
       "      <td>0.229738</td>\n",
       "      <td>0.517541</td>\n",
       "      <td>0.347157</td>\n",
       "      <td>0.949770</td>\n",
       "      <td>0.530748</td>\n",
       "      <td>2.542493e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>26557</td>\n",
       "      <td>1000</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>100</td>\n",
       "      <td>125</td>\n",
       "      <td>0.90</td>\n",
       "      <td>0.265902</td>\n",
       "      <td>0.434786</td>\n",
       "      <td>0.333768</td>\n",
       "      <td>0.881145</td>\n",
       "      <td>0.448620</td>\n",
       "      <td>1.715710e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>14599</td>\n",
       "      <td>1000</td>\n",
       "      <td>50</td>\n",
       "      <td>3</td>\n",
       "      <td>100</td>\n",
       "      <td>125</td>\n",
       "      <td>0.90</td>\n",
       "      <td>0.376291</td>\n",
       "      <td>0.385889</td>\n",
       "      <td>0.332385</td>\n",
       "      <td>0.829967</td>\n",
       "      <td>0.451762</td>\n",
       "      <td>1.645925e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>18485</td>\n",
       "      <td>1000</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>100</td>\n",
       "      <td>1000</td>\n",
       "      <td>0.80</td>\n",
       "      <td>0.252280</td>\n",
       "      <td>0.612480</td>\n",
       "      <td>0.327599</td>\n",
       "      <td>0.913583</td>\n",
       "      <td>0.655302</td>\n",
       "      <td>1.423714e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>23626</td>\n",
       "      <td>1000</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>100</td>\n",
       "      <td>500</td>\n",
       "      <td>0.90</td>\n",
       "      <td>0.177062</td>\n",
       "      <td>0.664016</td>\n",
       "      <td>0.326488</td>\n",
       "      <td>0.910068</td>\n",
       "      <td>0.535447</td>\n",
       "      <td>1.376154e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>19009</td>\n",
       "      <td>1000</td>\n",
       "      <td>50</td>\n",
       "      <td>3</td>\n",
       "      <td>100</td>\n",
       "      <td>1000</td>\n",
       "      <td>0.90</td>\n",
       "      <td>0.309680</td>\n",
       "      <td>0.402870</td>\n",
       "      <td>0.324568</td>\n",
       "      <td>0.847914</td>\n",
       "      <td>0.595794</td>\n",
       "      <td>1.297365e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>41301</td>\n",
       "      <td>1000</td>\n",
       "      <td>50</td>\n",
       "      <td>3</td>\n",
       "      <td>100</td>\n",
       "      <td>500</td>\n",
       "      <td>0.80</td>\n",
       "      <td>0.184869</td>\n",
       "      <td>0.581886</td>\n",
       "      <td>0.310396</td>\n",
       "      <td>0.952840</td>\n",
       "      <td>0.659371</td>\n",
       "      <td>8.301652e-06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>10890</td>\n",
       "      <td>1000</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>100</td>\n",
       "      <td>250</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.177108</td>\n",
       "      <td>0.568291</td>\n",
       "      <td>0.308822</td>\n",
       "      <td>0.958475</td>\n",
       "      <td>0.619231</td>\n",
       "      <td>7.890126e-06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>35388</td>\n",
       "      <td>1000</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>100</td>\n",
       "      <td>1000</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.157343</td>\n",
       "      <td>0.639666</td>\n",
       "      <td>0.293386</td>\n",
       "      <td>0.929776</td>\n",
       "      <td>0.734826</td>\n",
       "      <td>4.724903e-06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>43970</td>\n",
       "      <td>1000</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>100</td>\n",
       "      <td>500</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.135024</td>\n",
       "      <td>0.684492</td>\n",
       "      <td>0.259296</td>\n",
       "      <td>0.979487</td>\n",
       "      <td>0.743355</td>\n",
       "      <td>1.373931e-06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>04184</td>\n",
       "      <td>1000</td>\n",
       "      <td>50</td>\n",
       "      <td>3</td>\n",
       "      <td>100</td>\n",
       "      <td>1000</td>\n",
       "      <td>0.80</td>\n",
       "      <td>0.195156</td>\n",
       "      <td>0.698657</td>\n",
       "      <td>0.161388</td>\n",
       "      <td>0.868610</td>\n",
       "      <td>0.580306</td>\n",
       "      <td>1.198732e-08</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   pickle  dataset_size  n_epochs  seed  embedding_dimension  none_size  \\\n",
       "24  17784  1000          50         2    100                  250         \n",
       "19  43621  1000          50         2    100                  500         \n",
       "8   13437  1000          50         3    100                  250         \n",
       "7   47717  1000          50         3    100                  125         \n",
       "39  20445  1000          50         2    100                  125         \n",
       "29  20773  1000          50         3    100                  500         \n",
       "34  44204  1000          50         2    100                  125         \n",
       "33  38072  1000          50         2    100                  500         \n",
       "26  23894  1000          50         1    100                  250         \n",
       "27  32058  1000          50         2    100                  125         \n",
       "28  06771  1000          50         2    100                  125         \n",
       "5   09428  1000          50         3    100                  250         \n",
       "22  28109  1000          50         2    100                  250         \n",
       "13  26207  1000          50         1    100                  1000        \n",
       "18  03532  1000          50         3    100                  250         \n",
       "0   19224  1000          50         3    100                  125         \n",
       "21  10391  1000          50         1    100                  125         \n",
       "14  35230  1000          50         3    100                  500         \n",
       "12  32204  1000          50         3    100                  500         \n",
       "10  17684  1000          50         1    100                  250         \n",
       "1   42380  1000          50         1    100                  500         \n",
       "37  12842  1000          50         1    100                  250         \n",
       "9   37436  1000          50         1    100                  1000        \n",
       "2   02348  1000          50         1    100                  125         \n",
       "17  07207  1000          50         3    100                  125         \n",
       "23  43896  1000          50         2    100                  1000        \n",
       "6   44530  1000          50         1    100                  500         \n",
       "11  02708  1000          50         3    100                  1000        \n",
       "36  12484  1000          50         2    100                  1000        \n",
       "15  39873  1000          50         2    100                  250         \n",
       "16  26557  1000          50         1    100                  125         \n",
       "32  14599  1000          50         3    100                  125         \n",
       "20  18485  1000          50         2    100                  1000        \n",
       "25  23626  1000          50         2    100                  500         \n",
       "38  19009  1000          50         3    100                  1000        \n",
       "35  41301  1000          50         3    100                  500         \n",
       "30  10890  1000          50         2    100                  250         \n",
       "4   35388  1000          50         2    100                  1000        \n",
       "31  43970  1000          50         2    100                  500         \n",
       "3   04184  1000          50         3    100                  1000        \n",
       "\n",
       "    cosine_threshold  bleu_quality  bleu_diversity  accuracy  originality  \\\n",
       "24  0.95              0.393728      0.237461        0.717383  0.838710      \n",
       "19  0.95              0.432848      0.248355        0.645143  0.773662      \n",
       "8   0.00              0.333354      0.215302        0.639930  0.887567      \n",
       "7   0.95              0.435449      0.245025        0.580291  0.813852      \n",
       "39  0.95              0.421165      0.316455        0.527633  0.780232      \n",
       "29  0.00              0.367772      0.461277        0.503981  0.804166      \n",
       "34  0.90              0.478195      0.309747        0.498144  0.792718      \n",
       "33  0.80              0.194440      0.454097        0.494651  0.948659      \n",
       "26  0.80              0.395414      0.302146        0.489667  0.873030      \n",
       "27  0.80              0.381793      0.283795        0.488118  0.907601      \n",
       "28  0.00              0.255093      0.342376        0.479748  0.964264      \n",
       "5   0.95              0.334538      0.352572        0.478821  0.828161      \n",
       "22  0.90              0.343504      0.333271        0.468147  0.907099      \n",
       "13  0.90              0.293979      0.453825        0.459267  0.901299      \n",
       "18  0.90              0.392849      0.426362        0.452816  0.822553      \n",
       "0   0.80              0.273148      0.502153        0.443929  0.912004      \n",
       "21  0.95              0.338473      0.346794        0.432122  0.921764      \n",
       "14  0.95              0.321875      0.418420        0.420737  0.817312      \n",
       "12  0.90              0.417755      0.333813        0.417422  0.866784      \n",
       "10  0.90              0.335847      0.370672        0.414701  0.927364      \n",
       "1   0.90              0.357201      0.414022        0.403791  0.809686      \n",
       "37  0.95              0.277537      0.528968        0.382628  0.917369      \n",
       "9   0.95              0.310948      0.460927        0.380619  0.883294      \n",
       "2   0.80              0.388615      0.410206        0.379967  0.853646      \n",
       "17  0.00              0.369053      0.395023        0.379663  0.830798      \n",
       "23  0.95              0.294404      0.300528        0.357913  0.900580      \n",
       "6   0.95              0.228726      0.435705        0.356450  0.852910      \n",
       "11  0.95              0.276848      0.499742        0.355318  0.820321      \n",
       "36  0.90              0.182331      0.479930        0.348470  0.943802      \n",
       "15  0.80              0.229738      0.517541        0.347157  0.949770      \n",
       "16  0.90              0.265902      0.434786        0.333768  0.881145      \n",
       "32  0.90              0.376291      0.385889        0.332385  0.829967      \n",
       "20  0.80              0.252280      0.612480        0.327599  0.913583      \n",
       "25  0.90              0.177062      0.664016        0.326488  0.910068      \n",
       "38  0.90              0.309680      0.402870        0.324568  0.847914      \n",
       "35  0.80              0.184869      0.581886        0.310396  0.952840      \n",
       "30  0.00              0.177108      0.568291        0.308822  0.958475      \n",
       "4   0.00              0.157343      0.639666        0.293386  0.929776      \n",
       "31  0.00              0.135024      0.684492        0.259296  0.979487      \n",
       "3   0.80              0.195156      0.698657        0.161388  0.868610      \n",
       "\n",
       "    transfer        metric  \n",
       "24  0.493467  3.610037e-02  \n",
       "19  0.298874  1.248998e-02  \n",
       "8   0.539501  1.151660e-02  \n",
       "7   0.424630  4.329743e-03  \n",
       "39  0.431380  1.672334e-03  \n",
       "29  0.592593  1.057165e-03  \n",
       "34  0.464515  9.409081e-04  \n",
       "33  0.583499  8.769759e-04  \n",
       "26  0.509737  7.925198e-04  \n",
       "27  0.473487  7.678044e-04  \n",
       "28  0.589489  6.458545e-04  \n",
       "5   0.480083  6.334806e-04  \n",
       "22  0.464829  5.056167e-04  \n",
       "13  0.644951  4.174959e-04  \n",
       "18  0.479514  3.624281e-04  \n",
       "0   0.545504  2.972623e-04  \n",
       "21  0.469238  2.270175e-04  \n",
       "14  0.489134  1.738238e-04  \n",
       "12  0.544611  1.606015e-04  \n",
       "10  0.461646  1.504359e-04  \n",
       "1   0.463653  1.152314e-04  \n",
       "37  0.468548  6.726160e-05  \n",
       "9   0.542228  6.381312e-05  \n",
       "2   0.409726  6.272751e-05  \n",
       "17  0.490608  6.222711e-05  \n",
       "23  0.543948  3.449640e-05  \n",
       "6   0.515948  3.311180e-05  \n",
       "11  0.493750  3.207585e-05  \n",
       "36  0.654712  2.640272e-05  \n",
       "15  0.530748  2.542493e-05  \n",
       "16  0.448620  1.715710e-05  \n",
       "32  0.451762  1.645925e-05  \n",
       "20  0.655302  1.423714e-05  \n",
       "25  0.535447  1.376154e-05  \n",
       "38  0.595794  1.297365e-05  \n",
       "35  0.659371  8.301652e-06  \n",
       "30  0.619231  7.890126e-06  \n",
       "4   0.734826  4.724903e-06  \n",
       "31  0.743355  1.373931e-06  \n",
       "3   0.580306  1.198732e-08  "
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from pathlib import Path\n",
    "import shutil\n",
    "results_dir = 'results_grid_3'\n",
    "runs = []\n",
    "\n",
    "for filename in os.listdir(results_dir):\n",
    "    filepath = Path(results_dir)/Path(filename)\n",
    "    if filename.endswith('.pkl'):\n",
    "        runs.append(torch.load(filepath))\n",
    "        \n",
    "runs = [run for run in runs if (run['args']['dataset_size']==1000)]\n",
    "        \n",
    "data = {}\n",
    "for arg_name in runs[0]['args'].keys():\n",
    "    data[arg_name] = [run['args'][arg_name] for run in runs]\n",
    "data['pickle']   = np.array([run['args']['pickle'].split('/')[-1].rstrip('.pkl') for run in runs])\n",
    "    \n",
    "data['bleu_quality']   = np.array([run['delexicalised_metrics']['bleu_scores']['quality']['avg'] for run in runs])\n",
    "data['bleu_diversity'] = np.array([run['delexicalised_metrics']['bleu_scores']['diversity']['avg'] for run in runs])\n",
    "data['accuracy']       = np.array([run['delexicalised_metrics']['intent_accuracy']['avg'] for run in runs])\n",
    "data['originality']       = np.array([run['delexicalised_metrics']['originality']['avg'] for run in runs])\n",
    "data['transfer']   = np.array([run['delexicalised_metrics']['transfer']['metric']['avg'] for run in runs])\n",
    "\n",
    "data['metric'] = data['accuracy']**10 #* data['originality'] #* data['bleu_quality']\n",
    "\n",
    "df = pd.DataFrame(data, columns=['pickle', 'dataset_size', 'n_epochs', 'seed', 'embedding_dimension',#'x1', 'x2', 'k1', 'k2', \n",
    "                                 'none_size', 'cosine_threshold',\n",
    "                                 'bleu_quality', 'bleu_diversity', 'accuracy', 'originality', 'transfer', 'metric'])\n",
    "df = df.sort_values(by = 'metric', ascending=False)\n",
    "print(len(runs))\n",
    "df.head(1000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get all the seeds of the best run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('AddToPlaylist', 'add _artist_ to _playlist_owner_ _playlist_ list'),\n",
       " ('None',\n",
       "  'was it _weatherforecasttemperaturename_ in _weatherforecastcountry_ _weatherforecaststartdatetime_'),\n",
       " ('SearchCreativeWork', 'find the _object_type_ _object_name_ play on'),\n",
       " ('PlayMusic', 'play the _sort_ sort by _artist_ on _service_'),\n",
       " ('None', 'any chance it it will be _forecast_temperature_name_'),\n",
       " ('GetWeather', 'what are the weather forecast for _city_ _state_'),\n",
       " ('PlayMusic',\n",
       "  '_party_size_description_ play my _playlist_ playlist on _service_'),\n",
       " ('SearchScreeningEvent', 'what are the _object_type_ for _location_name_'),\n",
       " ('None', 'show me the weather forecast for the city of _city_'),\n",
       " ('AddToPlaylist', 'add this _music_item_ to _playlist_'),\n",
       " ('PlayMusic',\n",
       "  'play the _sort_ by _artist_ from _music_item_ from the _year_'),\n",
       " ('SearchScreeningEvent',\n",
       "  'is _movie_name_ being played at the _location_name_'),\n",
       " ('AddToPlaylist', 'add this _music_item_ to _playlist_'),\n",
       " ('SearchCreativeWork', 'i need find the _object_type_ called _object_name_'),\n",
       " ('PlayMusic',\n",
       "  'i want to hear something from the _year_ by _artist_ on _service_'),\n",
       " ('AddToPlaylist', 'add this _music_item_ to _playlist_'),\n",
       " ('SearchScreeningEvent',\n",
       "  \"i 'd like to watch _movie_name_ at a _restaurant_type_ _timerange_\"),\n",
       " ('AddToPlaylist', 'add _artist_ to _playlist_owner_ _playlist_ list'),\n",
       " ('BookRestaurant',\n",
       "  'find a reservation _spatial_relation_ from _poi_ _timerange_ for _party_size_number_ people at a _restaurant_type_ for _timerange_'),\n",
       " ('SearchCreativeWork', 'find the _object_name_ _object_type_')]"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "irun = np.argmax(data['metric'])\n",
    "print(irun)\n",
    "run = runs[irun]\n",
    "list(zip(run['generated']['intents'], runs[irun]['generated']['delexicalised']))[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'BookRestaurant': {'BookRestaurant': 4309,\n",
       "  'GetWeather': 520,\n",
       "  'PlayMusic': 0,\n",
       "  'RateBook': 37,\n",
       "  'SearchCreativeWork': 0,\n",
       "  'SearchScreeningEvent': 98,\n",
       "  'AddToPlaylist': 36,\n",
       "  'None': 0},\n",
       " 'GetWeather': {'BookRestaurant': 316,\n",
       "  'GetWeather': 4448,\n",
       "  'PlayMusic': 110,\n",
       "  'RateBook': 9,\n",
       "  'SearchCreativeWork': 0,\n",
       "  'SearchScreeningEvent': 107,\n",
       "  'AddToPlaylist': 10,\n",
       "  'None': 0},\n",
       " 'PlayMusic': {'BookRestaurant': 21,\n",
       "  'GetWeather': 0,\n",
       "  'PlayMusic': 4582,\n",
       "  'RateBook': 43,\n",
       "  'SearchCreativeWork': 0,\n",
       "  'SearchScreeningEvent': 47,\n",
       "  'AddToPlaylist': 307,\n",
       "  'None': 0},\n",
       " 'RateBook': {'BookRestaurant': 2,\n",
       "  'GetWeather': 4,\n",
       "  'PlayMusic': 0,\n",
       "  'RateBook': 4878,\n",
       "  'SearchCreativeWork': 40,\n",
       "  'SearchScreeningEvent': 72,\n",
       "  'AddToPlaylist': 4,\n",
       "  'None': 0},\n",
       " 'SearchCreativeWork': {'BookRestaurant': 0,\n",
       "  'GetWeather': 3,\n",
       "  'PlayMusic': 22,\n",
       "  'RateBook': 28,\n",
       "  'SearchCreativeWork': 4938,\n",
       "  'SearchScreeningEvent': 5,\n",
       "  'AddToPlaylist': 4,\n",
       "  'None': 0},\n",
       " 'SearchScreeningEvent': {'BookRestaurant': 51,\n",
       "  'GetWeather': 146,\n",
       "  'PlayMusic': 51,\n",
       "  'RateBook': 5,\n",
       "  'SearchCreativeWork': 93,\n",
       "  'SearchScreeningEvent': 4625,\n",
       "  'AddToPlaylist': 29,\n",
       "  'None': 0},\n",
       " 'AddToPlaylist': {'BookRestaurant': 0,\n",
       "  'GetWeather': 0,\n",
       "  'PlayMusic': 147,\n",
       "  'RateBook': 2,\n",
       "  'SearchCreativeWork': 0,\n",
       "  'SearchScreeningEvent': 4,\n",
       "  'AddToPlaylist': 4847,\n",
       "  'None': 0},\n",
       " 'None': {'BookRestaurant': 0,\n",
       "  'GetWeather': 0,\n",
       "  'PlayMusic': 0,\n",
       "  'RateBook': 0,\n",
       "  'SearchCreativeWork': 0,\n",
       "  'SearchScreeningEvent': 0,\n",
       "  'AddToPlaylist': 0,\n",
       "  'None': 0}}"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "run['logs']['dev']['classifications']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['results_grid/04810', 'results_grid/27623', 'results_grid/13167']\n",
      "done\n"
     ]
    }
   ],
   "source": [
    "ibest = np.argmax(data['metric'])\n",
    "iruns = [ibest]\n",
    "best = runs[ibest]\n",
    "best_args = [value for arg, value in best['args'].items() if arg not in ['seed','pickle']]\n",
    "\n",
    "for irun, run in enumerate(runs):\n",
    "    if irun==ibest:\n",
    "        continue\n",
    "    args = [value for arg, value in run['args'].items() if arg not in ['seed','pickle']]\n",
    "    #print(args)\n",
    "    if args == best_args:\n",
    "        iruns.append(irun)\n",
    "\n",
    "pickles = [runs[i]['args']['pickle'].rstrip('.pkl') for i in iruns]\n",
    "print(pickles)\n",
    "\n",
    "# Save selected runs\n",
    "\n",
    "savedir = results_dir + '/selected_runs_{}'.format(best['args']['dataset_size'])\n",
    "if os.path.exists(savedir):\n",
    "    ! sudo rm -r {savedir} # shutil.rmtree(savedir)\n",
    "! sudo mkdir {savedir}\n",
    "\n",
    "for pickle in pickles:\n",
    "    ! sudo cp -r {pickle} {savedir}\n",
    "    ! sudo cp {pickle}.pkl {savedir}\n",
    "\n",
    "print('done')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
